/*
 *  This file is part of WWWPal.
 *
 *  Copyright (C) 2003  Dr. John Punin, Department of Computer Science
 *  Rensselaer Polytechnic Institute
 *
 *  WWWPal is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  WWWPal is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Foobar; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include "rep_general_logml.h"
#include "GR_write_logml.h"

int
write_webgraph_logml_file(FILE *fp, GraphFrame *gf, WebStatPtr msn, 
			  int nelements);
int
write_node_webgraph(FILE *fp,GraphFrame *gf,WebStatPtr msn,PNODE v);

int
write_nodes_webgraph(FILE *fp,GraphFrame *gf,WebStatPtr msn);

int
write_edges_webgraph(FILE *fp,GraphFrame *gf,WebStatPtr msn);

int 
write_nodes_independent(FILE *fp,GraphFrame *gf,WebStatPtr msn);

int
write_edges_independent(FILE *fp,GraphFrame *gf,WebStatPtr msn);

int
write_hosts_logml_file(FILE *fp, WebStatPtr msn,int nelements);

int
write_domains_logml_file(FILE *fp, WebStatPtr msn);

int
write_directories_logml_file(FILE *fp, WebStatPtr msn,int nelements);

int
write_user_agents_logml_file(FILE *fp, WebStatPtr msn);

int
write_host_referers_logml_file(FILE *fp, WebStatPtr msn,int nelements);

int
write_referers_logml_file(FILE *fp, WebStatPtr msn,int nelements);

int
write_keywords_logml_file(FILE *fp, WebStatPtr msn,int nkeywords);

int
write_summary_logml_file(FILE *fp, GraphFrame *gf,WebStatPtr msn);

int
write_user_sessions_logml_file(FILE *fp, GraphFrame *gf,WebStatPtr msn,
			       int count,int max_edges,int min_edges);

int 
report_logs_logml(GraphFrame *gf, WebStatPtr msn, char *name_file,
		  int nelements,int nkeywords,int nusers,int max_edges,
		  int min_edges)
{
  FILE *fp = assert_open_xml_file(name_file);
  time_t start_date;
  time_t end_date;
  char sdate[100];
  char edate[100];
  
  short cus = LOGR_get_compute_user_sessions(msn);
  if(!fp)
      return -1;
  if(!msn)
      return -2;
  start_date = LOGR_get_minimum_start_date(msn);
  HTBufferTimeApache(start_date,sdate,100);
  end_date = LOGR_get_maximum_end_date(msn);
  HTBufferTimeApache(end_date,edate,100);
  GR_write_xml_head_logml_file(fp);
  GR_write_head_logml_file(fp,sdate,edate);
  write_webgraph_logml_file(fp,gf,msn,nelements);
  write_hosts_logml_file(fp,msn,nelements);
  write_domains_logml_file(fp,msn);
  write_directories_logml_file(fp,msn,nelements);
  write_user_agents_logml_file(fp,msn);
  write_host_referers_logml_file(fp,msn,nelements);
  write_referers_logml_file(fp,msn,nelements);
  write_keywords_logml_file(fp,msn,nkeywords);
  write_summary_logml_file(fp,gf,msn);
  if(cus)
      write_user_sessions_logml_file(fp,gf,msn,nusers,max_edges,min_edges);
  GR_write_end_logml_file(fp);
  fclose(fp);
  return 0;
}

int
write_webgraph_logml_file(FILE *fp, GraphFrame *gf, WebStatPtr msn, 
			  int nelements)
{
    GR_write_start_graph_logml_file(fp);
    write_nodes_webgraph(fp,gf,msn);    
    write_edges_webgraph(fp,gf,msn);    
    write_nodes_independent(fp,gf,msn);    
    write_edges_independent(fp,gf,msn);    
    GR_write_end_graph_logml_file(fp);
    return 0;
}

void
verify_web_nodes_reported(FILE *fp,GraphFrame *gf,WebStatPtr msn,
			  int sid,int tid)
{
    PNODE v = find_vertex_by_number(gf,sid);
    int hits = 0;
    if(v && !is_vertex_highlighted(v))
    {
	hits = get_counter(v);
	if(hits == 0) 
	{
	    write_node_webgraph(fp,gf,msn,v);
	    highlight_vertex(v,gf);
	}
    }
    v = find_vertex_by_number(gf,tid);
    if(v && !is_vertex_highlighted(v))
    {
	hits = get_counter(v);
	if(hits == 0)
	{
	    write_node_webgraph(fp,gf,msn,v);
	    highlight_vertex(v,gf);
	}
    }

}

int
write_edge_independent(FILE *fp,GraphFrame *gf,WebStatPtr msn,
			WebIndependentLinkPtr il)
{
  int sid = 0;
  int tid = 0;  
  int hits = 0;
  sid = LOGR_get_link_source_num(il);
  tid = LOGR_get_link_target_num(il);
  hits = LOGR_get_link_naccesses(il);
  verify_web_nodes_reported(fp,gf,msn,sid,tid);
  GR_write_empty_edge_logml_file(fp,sid,tid,NULL,hits,1);
  return 0;
}

int
write_edges_independent(FILE *fp,GraphFrame *gf,WebStatPtr msn)
{
  LNode_type *independent_links = LOGR_get_list_independent_links(msn);
  LNode_type *ptr = NULL;

  for(ptr = independent_links->back; ptr != independent_links; ptr=ptr->back)
  {
    WebIndependentLinkPtr il = (WebIndependentLinkPtr)ptr->info;
    write_edge_independent(fp,gf,msn,il);	
  }
  dehighlight_all_nodes(gf);
  return 0;	
}


int
write_node_attributes_independent_logml(FILE *fp,WebIndependentPagePtr mip)
{
    char *smime = (char *)LOGR_get_ip_mime(mip);
    long sbytes = LOGR_get_ip_sizebytes(mip);
    int httpcode = LOGR_get_ip_http_code(mip);
    char ssbytes[20],shttpcode[20];

    if(sbytes > 0)
    {
	sprintf(ssbytes,"%ld",sbytes);
	GR_write_node_att_logml_file(fp,"size",ssbytes);
    }

    if(httpcode > 0)
    {
	sprintf(shttpcode,"%d",httpcode);
	GR_write_node_att_logml_file(fp,"code",shttpcode);
    }

    if(smime)
	GR_write_node_att_logml_file(fp,"mime",smime);

    return 0;
}

int
write_node_independent(FILE *fp,GraphFrame *gf,WebStatPtr msn,
		       WebIndependentPagePtr mip)
{
    char *slabel = (char *)LOGR_get_ip_url(mip);
    char *xmllabel = slabel ? LOGR_get_xml_nescape_string(slabel) : NULL;
    int nid = LOGR_get_ip_num(mip);
    int nhits = LOGR_get_ip_naccesses(mip);
    int ehits = LOGR_get_entry_page_hits(msn,slabel);

    GR_write_start_node_logml_file(fp,nid,xmllabel,nhits,1,ehits);
    write_node_attributes_independent_logml(fp,mip);
    GR_write_end_node_logml_file(fp);

    if(xmllabel) {	
	free(xmllabel); xmllabel = NULL;
    }
    return 0;
}

int 
write_nodes_independent(FILE *fp,GraphFrame *gf,WebStatPtr msn)
{
    LNode_type *lips = LOGR_get_list_independent_pages(msn);
    LNode_type *ptr = lips->back;
    for(ptr = lips->back; ptr != lips; ptr=ptr->back)
    {
	WebIndependentPagePtr mip = (WebIndependentPagePtr)ptr->info;
	write_node_independent(fp,gf,msn,mip);
    }
    return 0;
}   

int
write_node_attributes_logml(FILE *fp,PNODE v)
{
    LNode_type *alist = get_attribute_list(v);
    LNode_type *ptr = NULL;
    char *aname = NULL;
    char *avalue = NULL;
    char *xmllabel = NULL; 

    if(!alist)
	return 0;

    for(ptr = alist->back; ptr != alist; ptr = ptr->back)
    {
	get_node_info_assoc_list(ptr,&aname,&avalue);

	if(!strcmp(aname,"title")) 
	{
	    xmllabel = avalue ? LOGR_get_xml_string(avalue) : NULL;
	    avalue = xmllabel;
	}
	else if(!strcmp(aname,"content")) 
	{
	    xmllabel = avalue ? LOGR_get_xml_string(avalue) : NULL;
	    avalue = xmllabel;
	}

	if(aname && avalue)
	    GR_write_node_att_logml_file(fp,aname,avalue);

	if(xmllabel) {
	    free(xmllabel); xmllabel = NULL;
	}
    }

    return 0;
}

int
write_node_webgraph(FILE *fp,GraphFrame *gf,WebStatPtr msn,PNODE v)
{
    char *slabel = get_label_vertex(v);	
    char *xmllabel = slabel ? LOGR_get_xml_string(slabel) : NULL;
    int nid = get_vertex_number(v);
    int nhits = get_counter(v);
    int ehits = LOGR_get_entry_page_hits(msn,slabel);

    GR_write_start_node_logml_file(fp,nid,xmllabel,nhits,0,ehits);
    write_node_attributes_logml(fp,v);
    GR_write_end_node_logml_file(fp);
    if(xmllabel) {	
	free(xmllabel); xmllabel = NULL;
    }
    return 0;
}

int
write_nodes_webgraph(FILE *fp,GraphFrame *gf,WebStatPtr msn)
{
  LNode_type *nodes = get_list_vertices(gf);
  LNode_type *ptr = nodes->back;

  for(; ptr != nodes; ptr=ptr->back)
  {
    PNODE v = (PNODE)ptr->info;
    int hits = get_counter(v);
    if(hits > 0)
	write_node_webgraph(fp,gf,msn,v);
  }
  return 0;
}

int
write_edge_webgraph(FILE *fp,GraphFrame *gf,WebStatPtr msn,PEDGE e)
{
    int sid = 0;
    int tid = 0;
    char *elabel = NULL;
    int hits = 0;
    char att = '\0';
    char *satt = NULL;
    char *we = NULL;
    char *xmlesclabel = NULL;
    
    att = get_edge_attribute(e);

    sid = get_vertex_number(get_from_vertex(e));
    tid = get_vertex_number(get_to_vertex(e));
    elabel = get_label_edge(e);
    we = get_weight_edge(e);
    hits = we ? atoi(we) : 0;
    xmlesclabel = elabel ? LOGR_get_xml_nescape_string(elabel): NULL;

    verify_web_nodes_reported(fp,gf,msn,sid,tid);
    
    if(att == '\0')
	GR_write_empty_edge_logml_file(fp,sid,tid,xmlesclabel,hits,0);
    else
    {
	satt = get_string_edge_attribute(e);
	GR_write_start_edge_logml_file(fp,sid,tid,xmlesclabel,hits,0);
	GR_write_att_edge_logml_file(fp,satt);
	GR_write_end_edge_logml_file(fp);
	free(satt); satt = NULL;
    }

    if(xmlesclabel) {	
	free(xmlesclabel); xmlesclabel = NULL;
    }

    return 0;
}

int
write_edges_webgraph(FILE *fp,GraphFrame *gf,WebStatPtr msn)
{
    LNode_type *nodes = get_list_vertices(gf);
    LNode_type *ptr = nodes->back;
    
    for(; ptr != nodes; ptr=ptr->back)
    {
      PNODE v = (PNODE)ptr->info;
      LNode_type *peo, *ledges_out = get_edges_out(v);
      for(peo = ledges_out->back ; peo != ledges_out ; peo = peo->back)
	{
	  PEDGE e = (PEDGE)peo->info;
	  char *we = get_weight_edge(e);
	  int cedge = we ? atoi(we) : 0;

	  if(cedge > 0)
	  {
	      write_edge_webgraph(fp,gf,msn,e);
	  }
	}
    }
  return 0;
}

void **
get_array_from_list(LNode_type *list, int *nele)
{
    LNode_type *ptr = NULL;
    void **varray = NULL;
    int i = 0;
    int n = list ? count_llist(list) : 0;

    varray = n ? (void **)calloc(n,sizeof(void *)) : NULL;

    if(!varray && n > 0) {
      fprintf(stderr,"Out of Memory get_array_from_list\n");
      abort();
    }

    for(ptr = list->back; ptr != list; ptr=ptr->back)
    {
	void * info = (void *)ptr->info;
	varray[i++] = info;
    }

    *nele = n;
    return varray;
}



int
write_host_logml_file(FILE *fp, WebStatPtr msn, WebSiteNodePtr msite)
{
    char *lsite = (char *)LOGR_get_site_dns(msite);
    char *ipsite= (char *)LOGR_get_site_ip(msite);
    long scount = LOGR_get_site_naccesses(msite);
    long ssize = LOGR_get_site_sizebytes(msite);
    long npages = LOGR_get_site_nacc_search(msite);
    int is_internal = 0;

    if(!lsite && !ipsite)
	return 0;

    if(lsite)
	is_internal = LOGR_is_address_local(msn,lsite);
    else if(ipsite)
	is_internal = LOGR_is_address_local(msn,ipsite);

    if(ipsite && lsite)
	GR_write_host_logml_file(fp,lsite,ipsite,scount,ssize,npages,
				 is_internal);
    else if(ipsite)
	GR_write_host_logml_file(fp,NULL,ipsite,scount,ssize,npages,
				 is_internal);
    else if(lsite)
	GR_write_host_logml_file(fp,lsite,NULL,scount,ssize,npages,
				 is_internal);

    return 0;
}

int
write_hosts_logml_file(FILE *fp, WebStatPtr msn,int nelements)
{
    WebSiteNodePtr *varray = NULL;
    LNode_type *lsites = LOGR_get_list_sites(msn);
    int i = 0,tn = 0,n = 0;

    varray = (WebSiteNodePtr *)get_array_from_list(lsites,&n);

    tn = n < nelements ? n : nelements;

    GR_write_start_hosts_logml_file(fp, tn);
    
    for(i = 1; i<=tn; i++)
    {
	WebSiteNodePtr msite= (WebSiteNodePtr)
	    randomized_select((GPTR *)varray,i-1,n-1,1,
			      LOGR_is_access_lower,LOGR_is_access_bigger);
	if(msite)
	{
	    write_host_logml_file(fp, msn, msite);
	}
    }

    GR_write_end_hosts_logml_file(fp);

    free(varray); varray = NULL;
    return 0;
}

int
write_domain_logml_file(FILE *fp, WebStatPtr msn,WebGenInfoPtr mgi)
{
    const char *lcode = (char *)LOGR_get_gen_info_context(mgi);
    long scount = LOGR_get_gen_info_naccesses(mgi);   
    double ssize = LOGR_get_gen_info_sizebytes(mgi);
    long npages = LOGR_get_gen_info_npages(mgi); 
    char str_size[30];
    sprintf(str_size,"%.0f",ssize);

    if(!lcode)
	return 0;

    GR_write_domain_logml_file(fp,(char *)lcode,scount,str_size,npages);

    return 0;
}

int
write_domains_logml_file(FILE *fp, WebStatPtr msn)
{
    LNode_type *ldomains = LOGR_get_list_domains(msn);
    WebGenInfoPtr *varray = NULL;

    int i = 0,tn = 0;

    varray = (WebGenInfoPtr *)get_array_from_list(ldomains,&tn);

    if(tn <= 0)	
	return -1;

    GR_write_start_domains_logml_file(fp, tn);
    
    for(i = 1; i<=tn; i++)
    {
	WebGenInfoPtr mgi= (WebGenInfoPtr)
	    randomized_select((GPTR *)varray,i-1,tn-1,1,
			      LOGR_is_access_gen_info_lower,
			      LOGR_is_access_gen_info_bigger);
	if(mgi)
	{
	    write_domain_logml_file(fp, msn, mgi);
	}
    }

    GR_write_end_domains_logml_file(fp);

    free(varray); varray = NULL;
    return 0;
}

int
write_directory_logml_file(FILE *fp, WebStatPtr msn, WebGenPagePtr mdir)
{

    char *sdir = (char *)LOGR_get_info_gen_page(mdir);
    char *xmlesclabel = sdir ? LOGR_get_xml_nescape_string(sdir): NULL;
    long scount = LOGR_get_gen_nacc_static(mdir);
    long tcount = LOGR_get_gen_naccesses(mdir);
    long ssize = LOGR_get_gen_total_bytes(mdir);

    if(!sdir)
	return 0;

    GR_write_directory_logml_file(fp,xmlesclabel,scount,tcount,ssize);

    if(xmlesclabel) {	
	free(xmlesclabel); xmlesclabel = NULL;
    }	

    return 0;
}

int
write_directories_logml_file(FILE *fp, WebStatPtr msn,int nelements)
{
    WebGenPagePtr *varray = NULL;
    LNode_type *ldir = LOGR_get_list_directories(msn);
    int i = 0,tn = 0,n = 0;

    varray = (WebGenPagePtr *)get_array_from_list(ldir,&n);

    tn = n < nelements ? n : nelements;

    GR_write_start_directories_logml_file(fp, tn);
    
    for(i = 1; i<=tn; i++)
    {
	WebGenPagePtr mdir= (WebGenPagePtr)
	    randomized_select((GPTR *)varray,i-1,n-1,1,
			      LOGR_is_access_stat_gen_lower,
			      LOGR_is_access_stat_gen_bigger);
	if(mdir)
	{
	    write_directory_logml_file(fp, msn, mdir);
	}
    }

    GR_write_end_directories_logml_file(fp);

    free(varray); varray = NULL;
    return 0;
}

int
write_user_agent_logml_file(FILE *fp, WebStatPtr msn, WebBrowserNodePtr mua)
{
    char *lbrowser = (char *)LOGR_get_user_agent(mua);
    char *xmlesclabel = lbrowser ? LOGR_get_xml_nescape_string(lbrowser): NULL;
    long scount = LOGR_get_browser_naccesses(mua);
    long nhtml = LOGR_get_browser_npages(mua);
    double ssize = LOGR_get_browser_sizebytes(mua);

    if(!lbrowser)
	return 0;

    GR_write_user_agent_logml_file(fp,xmlesclabel,scount,ssize,nhtml);

    if(xmlesclabel) {	
	free(xmlesclabel); xmlesclabel = NULL;
    }

    return 0;
}

int
write_user_agents_logml_file(FILE *fp, WebStatPtr msn)
{
    WebBrowserNodePtr *varray = NULL;
    LNode_type *luas = LOGR_get_list_ua(msn);

    int i = 0,tn = 0,n = 0;

    varray = (WebBrowserNodePtr *)get_array_from_list(luas,&n);

    if(n==0)
	return 0;

    tn = n;

    GR_write_start_user_agents_logml_file(fp, tn);
    
    for(i = 1; i<=tn; i++)
    {
	WebBrowserNodePtr mua= (WebBrowserNodePtr)
	    randomized_select((GPTR *)varray,i-1,n-1,1,
			      LOGR_is_accessb_lower,LOGR_is_accessb_bigger);
	if(mua)
	{
	    write_user_agent_logml_file(fp, msn, mua);
	}
    }

    GR_write_end_user_agents_logml_file(fp);

    free(varray); varray = NULL;
    return 0;
}

int
write_host_referer_logml_file(FILE *fp, WebStatPtr msn, WebSiteNodePtr mhref,
			      int is_search_engine)
{
    char *lsite = (char *)LOGR_get_site_dns(mhref);
    long scount = LOGR_get_site_naccesses(mhref);
    long scount_search = LOGR_get_site_nacc_search(mhref);
    long ssize = LOGR_get_site_sizebytes(mhref);
    long ssize_search = LOGR_get_site_sizebytes_search(mhref);
    long sac = scount-scount_search;
    long sbytes = ssize-ssize_search;

    if(!lsite)
	return 0;

    if(is_search_engine == 0)
	GR_write_host_referer_logml_file(fp,lsite,sac,sbytes,0);
    else
      GR_write_host_referer_logml_file(fp,lsite,scount_search,ssize_search,1);


    return 0;
}

int
write_host_referers_logml_file(FILE *fp, WebStatPtr msn,int nelements)
{
    WebSiteNodePtr *varray = NULL;
    WebSiteNodePtr *varray2 = NULL;
    LNode_type *lsites_ref = LOGR_get_list_sites_ref(msn);
    LNode_type *ptr = NULL;
    int i = 0,j = 0, tn = 0,tnse = 0;
    int tnrefsengine = 0;
    int tnrefhost = 0;

    for(ptr = lsites_ref->back; ptr != lsites_ref; ptr=ptr->back)
    {
	WebSiteNodePtr msite = (WebSiteNodePtr)ptr->info;
	TypeReferer tr = LOGR_get_site_type_referer(msite);
	if(tr & RDYNAMIC_REF) 
	    tnrefsengine++;
	if (tr == RNOREFERER)
	    tnrefhost++;
	if (tr & RSTATIC_REF)
	    tnrefhost++;
    }

    varray = tnrefhost ? (WebSiteNodePtr *)calloc(tnrefhost,sizeof(WebSiteNodePtr)) : NULL;
    
    if(!varray && tnrefhost > 0) {
	fprintf(stderr,"Out of Memory write_referers_logml_file\n");
	abort();
    }

    varray2 = tnrefsengine ? (WebSiteNodePtr *)calloc(tnrefsengine,sizeof(WebSiteNodePtr)) : NULL;
    
    if(!varray2 && tnrefsengine > 0) {
	fprintf(stderr,"Out of Memory write_referers_logml_file\n");
	abort();
    }

    for(ptr = lsites_ref->back; ptr != lsites_ref; ptr=ptr->back)
    {
	WebSiteNodePtr msite = (WebSiteNodePtr)ptr->info;
	TypeReferer tr = LOGR_get_site_type_referer(msite);
	if(tr & RDYNAMIC_REF)
	    varray2[j++] = msite;
	if (tr == RNOREFERER)
	    varray[i++] = msite;
	if (tr & RSTATIC_REF)
	    varray[i++] = msite;
    }

    tn = tnrefhost < nelements ? tnrefhost : nelements;
    tnse = tnrefsengine < nelements ? tnrefsengine : nelements;

    GR_write_start_host_referers_logml_file(fp, tn+tnse);

    for(i = 1; i<=tn; i++)
    {
	WebSiteNodePtr msite= (WebSiteNodePtr)
	    randomized_select((GPTR *)varray,i-1,tnrefhost-1,1,
		       LOGR_is_access_stat_lower,LOGR_is_access_stat_bigger);
	if(msite)
	{
	    write_host_referer_logml_file(fp, msn, msite,0);
	}
    }

    tn = tnrefsengine < nelements ? tnrefsengine : nelements;

    for(i = 1; i<=tn; i++)
    {
	WebSiteNodePtr msite= (WebSiteNodePtr)
	    randomized_select((GPTR *)varray2,i-1,tnrefsengine-1,1,
        	    LOGR_is_access_search_lower,LOGR_is_access_search_bigger);
	if(msite)
	{
	    write_host_referer_logml_file(fp, msn, msite,1);
	}
    }

    GR_write_end_host_referers_logml_file(fp);

    free(varray); varray = NULL;
    free(varray2); varray2 = NULL;

    return 0;
}

int
write_external_referer_logml_file(FILE *fp, WebStatPtr msn, WebIndependentPagePtr ip)
{
    char *url = (char *)LOGR_get_ip_url(ip);
    char *xmllabel = url ? LOGR_get_xml_nescape_string(url) : NULL;
    long nacc = LOGR_get_ip_naccesses(ip);
    long ssize = LOGR_get_ip_sizebytes(ip);
    int http_code = LOGR_get_ip_http_code(ip);
    WebGenPagePtr gpn = (WebGenPagePtr)LOGR_get_ip_info(ip);
    int target = gpn ? LOGR_get_gen_num(gpn) : -1;

    if(!url)
	return 0;
    
    if(http_code == 200)
       GR_write_referer_logml_file(fp,xmllabel,nacc,ssize,0,target,-1);
    else
       GR_write_referer_logml_file(fp,xmllabel,nacc,ssize,0,target,http_code);

    if(xmllabel) {	
	free(xmllabel); xmllabel = NULL;
    }
    return 0;
}

int
write_internal_referer_logml_file(FILE *fp, WebStatPtr msn, WebGenPagePtr gpn)
{
    char *url = (char *)LOGR_get_gen_url(gpn);
    char *xmllabel = url ? LOGR_get_xml_string(url) : NULL;
    long nacc = LOGR_get_gen_naccesses(gpn);
    long ssize = LOGR_get_gen_total_bytes(gpn);

    if(!url)
	return 0;

    GR_write_referer_logml_file(fp,xmllabel,nacc,ssize,1,-1,-1);

    if(xmllabel) {	
	free(xmllabel); xmllabel = NULL;
    }
    return 0;
}

void **
get_array_error_from_list(LNode_type *list, int *nele)
{
    LNode_type *ptr = NULL;
    void **varray = NULL;
    int i = 0;
    int n = 0;

    for(ptr=list->back; ptr!=list; ptr=ptr->back)
    {
	WebIndependentPagePtr ip = (WebIndependentPagePtr)ptr->info;
	if(LOGR_get_ip_http_code(ip)>=400)
	    n++;
    }
    
    varray = n ? (void **)calloc(n,sizeof(void *)) : NULL;

    if(!varray && n > 0) {
      fprintf(stderr,"Out of Memory get_array_error_from_list\n");
      abort();
    }

    i=0;
    for(ptr = list->back; ptr != list; ptr=ptr->back)
    {
	WebIndependentPagePtr ip = (WebIndependentPagePtr)ptr->info;
	if(LOGR_get_ip_http_code(ip)>=400)
	    varray[i++] = (void *)ip;
    }

    *nele = n;
    return varray;
}

int
write_referers_logml_file(FILE *fp, WebStatPtr msn,int nelements)
{

    LNode_type *lurl_ext_ref = LOGR_get_list_url_external_referers(msn);
    LNode_type *lurl_int_ref = LOGR_get_list_url_internal_referers(msn);
    WebIndependentPagePtr *varray = NULL;
    WebGenPagePtr *varray2 = NULL;
    WebIndependentPagePtr *varray3 = NULL;

    int i = 0,tn = 0,tm = 0, n = 0,m = 0,tl = 0, l=0;

    varray = (WebIndependentPagePtr *)get_array_from_list(lurl_ext_ref,&n);
    varray2 = (WebGenPagePtr *)get_array_from_list(lurl_int_ref,&m);
    varray3 = (WebIndependentPagePtr *)
	get_array_error_from_list(lurl_ext_ref,&l);

    tn = n < nelements ? n : nelements;
    tm = m < nelements ? m : nelements;
    tl = l < nelements ? l : nelements;

    GR_write_start_referers_logml_file(fp, tn+tm+tl);
    
    for(i = 1; i<=tn; i++)
    {
	WebIndependentPagePtr opn = (WebIndependentPagePtr)
	    randomized_select((GPTR *)varray,i-1,n-1,1,
			   LOGR_is_access_ip_lower,LOGR_is_access_ip_bigger);

	if(opn)
	{
	    write_external_referer_logml_file(fp, msn,opn);
	}
    }

    for(i = 1; i<=tm ; i++)
    {
	WebGenPagePtr gpn = (WebGenPagePtr)
	    randomized_select((GPTR *)varray2,i-1,m-1,1,
			  LOGR_is_access_gen_lower,LOGR_is_access_gen_bigger);
	if(gpn)
	{
	    write_internal_referer_logml_file(fp,msn,gpn);
	}
    }

    for(i = 1; i<=tl; i++)
    {
	WebIndependentPagePtr opn = (WebIndependentPagePtr)
	    randomized_select((GPTR *)varray3,i-1,l-1,1,
			   LOGR_is_access_ip_lower,LOGR_is_access_ip_bigger);

	if(opn)
	{
	    write_external_referer_logml_file(fp,msn,opn);
	}
    }

    GR_write_end_referers_logml_file(fp);

    free(varray); varray = NULL;
    free(varray2); varray2 = NULL;
    free(varray3); varray3 = NULL;
    return 0;
}

int
write_keyword_logml_file(FILE *fp, WebStatPtr msn,WebKeywordPtr mk, int se)
{
    char *keyword = (char *)LOGR_get_keyword(mk);
    char *xmlesclabel = keyword ? LOGR_get_xml_nescape_string(keyword): NULL;
    int tcount = se ? LOGR_get_search_engine_count(mk,se) : 
	LOGR_get_total_count(mk);

    if(!keyword)
	return 0;

    GR_write_keyword_logml_file(fp,xmlesclabel,tcount);

    if(xmlesclabel) {	
	free(xmlesclabel); xmlesclabel = NULL;
    }

    return 0;
}

int
write_keywords_logml_file_by_search_engine(FILE *fp, WebStatPtr msn,
			     int nkeywords, char *name_search_engine, int se,
			      LNode_type *lkeys,int (*isl)(),int (*isb)())
{
    LNode_type *ptr = NULL;
    WebKeywordPtr *varray = NULL;
    int tn = 0;
    int tnkeys = 0;
    int i = 0;
    long tsearch = LOGR_get_total_number_searches(msn);

    for(ptr = lkeys->back; ptr != lkeys; ptr=ptr->back)
    {
	WebKeywordPtr mk = (WebKeywordPtr)ptr->info;
	int count = LOGR_get_search_engine_count(mk,se);
	if(count > 0)
	    tnkeys++;
    }

    varray = tnkeys ? (WebKeywordPtr *)calloc(tnkeys,sizeof(WebKeywordPtr)) : NULL;

  if(!varray && tnkeys > 0) {
      fprintf(stderr,"Out of Memory write_keywords_logml_file_by_search_engine\n");
      abort();
  }

  for(ptr = lkeys->back; ptr != lkeys; ptr=ptr->back)
  {
    WebKeywordPtr mk = (WebKeywordPtr)ptr->info;
    int count = LOGR_get_search_engine_count(mk,se);
    if(count > 0)
	varray[i++] = mk;
  }

  if(tnkeys == 0)
      return 0;

  tn = tnkeys < nkeywords ? tnkeys : nkeywords;
      
  GR_write_start_keywords_logml_file(fp, tn,name_search_engine,tsearch);

  for(i = 1; i<=tn ; i++)
  {
    WebKeywordPtr mk = (WebKeywordPtr)
	randomized_select((GPTR *)varray,i-1,tnkeys-1,1,isl,isb);
    if(mk)
    {
	write_keyword_logml_file(fp, msn, mk,se);
    }
  }

  GR_write_end_keywords_logml_file(fp);	

  free(varray); varray = NULL;

  return 0;
}


int
write_keywords_logml_file(FILE *fp, WebStatPtr msn,int nkeywords)
{
    WebKeywordPtr *varray = NULL;
    Hashtable_type *key_table = msn ? LOGR_get_table_keywords(msn) : NULL;
    LNode_type *lkeys = key_table ? get_list_elements(*key_table) : NULL;
    long tsearch = LOGR_get_total_number_searches(msn);
    
    int i = 0,tn = 0,n = 0;

    varray = (WebKeywordPtr *)get_array_from_list(lkeys,&n);

    tn = n < nkeywords ? n : nkeywords;

    if(n == 0)
      return 0;

    GR_write_start_keywords_logml_file(fp, tn,NULL,tsearch);
    
    for(i = 1; i<=tn; i++)
    {
	WebKeywordPtr mk = (WebKeywordPtr)
	    randomized_select((GPTR *)varray,i-1,n-1,1,
			  LOGR_is_total_count_keyword_lower,
			  LOGR_is_total_count_keyword_bigger);
	if(mk)
	{
	    write_keyword_logml_file(fp, msn, mk,0);
	}
    }

    GR_write_end_keywords_logml_file(fp);

    free(varray); varray = NULL;
    
    write_keywords_logml_file_by_search_engine(fp,msn,nkeywords,
					       SALTAVISTA, TALTAVISTA,lkeys,
	   LOGR_is_se_count1_keyword_lower,LOGR_is_se_count1_keyword_bigger);

    write_keywords_logml_file_by_search_engine(fp,msn,nkeywords,
					       SYAHOO, TYAHOO,lkeys,
	   LOGR_is_se_count2_keyword_lower,LOGR_is_se_count2_keyword_bigger);

    write_keywords_logml_file_by_search_engine(fp,msn,nkeywords,
					       SYAHOO2, TYAHOO2,lkeys,
	   LOGR_is_se_count3_keyword_lower,LOGR_is_se_count3_keyword_bigger);

    write_keywords_logml_file_by_search_engine(fp,msn,nkeywords,
					       SGOOGLE, TGOOGLE,lkeys,
	   LOGR_is_se_count4_keyword_lower,LOGR_is_se_count4_keyword_bigger);

    write_keywords_logml_file_by_search_engine(fp,msn,nkeywords,
					       SGOOGLE2, TGOOGLE2,lkeys,
	   LOGR_is_se_count5_keyword_lower,LOGR_is_se_count5_keyword_bigger);

    write_keywords_logml_file_by_search_engine(fp,msn,nkeywords,
					       SINFOSEEK, TINFOSEEK,lkeys,
	   LOGR_is_se_count6_keyword_lower,LOGR_is_se_count6_keyword_bigger);

    write_keywords_logml_file_by_search_engine(fp,msn,nkeywords,
					       SLYCOS, TLYCOS,lkeys,
	   LOGR_is_se_count7_keyword_lower,LOGR_is_se_count7_keyword_bigger);

    return 0;
}

int
write_http_code_logml_file(FILE *fp,WebStatPtr msn,WebGenInfoPtr gn)
{
    long npages = LOGR_get_gen_info_npages(gn);
    int code = (int)LOGR_get_gen_info_context(gn);
    double sbytes = LOGR_get_gen_info_sizebytes(gn);
    long acc = LOGR_get_gen_info_naccesses(gn);
    char *str = (char *)LOGR_get_string_code(code);

    GR_write_http_code_logml_file(fp,code,str,acc,sbytes,npages);

    return 0;
}

int
write_http_codes_logml_file(FILE *fp,WebStatPtr msn)
{
    LNode_type *lcodes = LOGR_get_alist_codes(msn);
    WebGenInfoPtr *varray = NULL;
    int i = 0,n = 0;

    varray = (WebGenInfoPtr *)get_array_from_list(lcodes,&n);

    for(i = 1; i<=n ; i++)
    {	
	WebGenInfoPtr gn = (WebGenInfoPtr)randomized_select((GPTR *)varray,
							    i-1,n-1,1,
					     LOGR_is_context_gen_info_bigger,
   				             LOGR_is_context_gen_info_lower);

	write_http_code_logml_file(fp,msn,gn);

    }

    free(varray); varray = NULL;
    return 0;
}


int
write_http_method_logml_file(FILE *fp,WebStatPtr msn,WebGenInfoPtr gn)
{
    long npages = LOGR_get_gen_info_npages(gn);
    int method = (int)LOGR_get_gen_info_context(gn);
    double sbytes = LOGR_get_gen_info_sizebytes(gn);
    long acc = LOGR_get_gen_info_naccesses(gn);
    char *str = (char *)LOGR_get_string_method(method);

    GR_write_http_method_logml_file(fp,str,acc,sbytes,npages);

    return 0;
}

int
write_http_methods_logml_file(FILE *fp,WebStatPtr msn)
{
    LNode_type *lmethods = LOGR_get_alist_methods(msn);
    WebGenInfoPtr *varray = NULL;
    int i = 0,n = 0;

    varray = (WebGenInfoPtr *)get_array_from_list(lmethods,&n);

    for(i = 1; i<=n ; i++)
    {	
	WebGenInfoPtr gn = (WebGenInfoPtr)randomized_select((GPTR *)varray,
							    i-1,n-1,1,
					     LOGR_is_context_gen_info_bigger,
   				             LOGR_is_context_gen_info_lower);

	write_http_method_logml_file(fp,msn,gn);

    }

    free(varray); varray = NULL;
    return 0;
}

int
write_http_version_logml_file(FILE *fp,WebStatPtr msn,WebGenInfoPtr gn)
{
    long npages = LOGR_get_gen_info_npages(gn);
    int version = (int)LOGR_get_gen_info_context(gn);
    double sbytes = LOGR_get_gen_info_sizebytes(gn);
    long acc = LOGR_get_gen_info_naccesses(gn);
    char *str = (char *)LOGR_get_token_version(version);

    GR_write_http_version_logml_file(fp,str,acc,sbytes,npages);

    return 0;
}

int
write_http_versions_logml_file(FILE *fp,WebStatPtr msn)
{
    LNode_type *lversions = LOGR_get_alist_versions(msn);
    WebGenInfoPtr *varray = NULL;
    int i = 0,n = 0;

    varray = (WebGenInfoPtr *)get_array_from_list(lversions,&n);

    for(i = 1; i<=n ; i++)
    {	
	WebGenInfoPtr gn = (WebGenInfoPtr)randomized_select((GPTR *)varray,
							    i-1,n-1,1,
					     LOGR_is_context_gen_info_bigger,
   				             LOGR_is_context_gen_info_lower);

	write_http_version_logml_file(fp,msn,gn);

    }

    free(varray); varray = NULL;
    return 0;
}

int
write_month_stat_logml_file(FILE *fp,WebStatPtr msn,WebGenInfoPtr wgi,
			    int month)
{
    long nacc = LOGR_get_gen_info_naccesses(wgi);
    long nhtml = LOGR_get_gen_info_npages(wgi);
    double bytes = LOGR_get_gen_info_sizebytes(wgi);
    if(nacc > 0)
	GR_write_month_stat_logml_file(fp,month,nacc,bytes,nhtml);

    return 0;
}

int
write_day_stat_logml_file(FILE *fp,WebStatPtr msn,WebGenInfoPtr wgi,
			  int day)
{
    long nacc = LOGR_get_gen_info_naccesses(wgi);
    long nhtml = LOGR_get_gen_info_npages(wgi);
    double bytes = LOGR_get_gen_info_sizebytes(wgi);

    if(nacc > 0)
	GR_write_day_stat_logml_file(fp,day,nacc,bytes,nhtml);

    return 0;
}

int
write_hour_stat_logml_file(FILE *fp,WebStatPtr msn,WebGenInfoPtr wgi,
			   int hour)
{
    long nacc = LOGR_get_gen_info_naccesses(wgi);
    long nhtml = LOGR_get_gen_info_npages(wgi);
    double bytes = LOGR_get_gen_info_sizebytes(wgi);

    if(nacc > 0)
	GR_write_hour_stat_logml_file(fp,hour,nacc,bytes,nhtml);

    return 0;
}

int
write_date_stats_logml_file(FILE *fp,WebStatPtr msn)
{
    int i = 0,n = 12;

    GR_write_start_date_stat_logml_file(fp);

    for(i = 0; i< n ; i++)
    {	 
	WebGenInfoPtr wgi = (WebGenInfoPtr)LOGR_get_info_months(msn,i);
	if(wgi)
	{
	    write_month_stat_logml_file(fp,msn,wgi,i+1);
	}
    }
    n = 31;
    for(i = 0 ; i< n ; i++)
    {	 
	WebGenInfoPtr wgi = (WebGenInfoPtr)LOGR_get_info_days(msn,i);
	if(wgi)
	{
	    write_day_stat_logml_file(fp,msn,wgi,i+1);
	}
    }

    n = 24;
    for(i = 0; i< n ; i++)
    {	 
	WebGenInfoPtr wgi = (WebGenInfoPtr)LOGR_get_info_hours(msn,i);
	if(wgi)
	{
	    write_hour_stat_logml_file(fp,msn,wgi,i);
	}
    }
    GR_write_end_date_stat_logml_file(fp);

    return 0;
}


int
write_summary_logml_file(FILE *fp, GraphFrame *gf,WebStatPtr msn)
{
  LNode_type *lsites = LOGR_get_list_sites(msn);
  LNode_type *lurl_ext_ref = LOGR_get_list_url_external_referers(msn);
  LNode_type *lurl_int_ref = LOGR_get_list_url_internal_referers(msn);
  int tnurlextref = lurl_ext_ref ? count_llist(lurl_ext_ref) : 0;
  int tnurlintref = lurl_int_ref ? count_llist(lurl_int_ref) : 0;

  LNode_type *luas = LOGR_get_list_ua(msn);
  /*LNode_type *lspiders = LOGR_get_list_spider(msn);*/
  int tnuas = luas ? count_llist(luas) : 0;
  /*int tnspiders = lspiders ? count_llist(lspiders) : 0;*/

  Hashtable_type *key_table = msn ? LOGR_get_table_keywords(msn) : NULL;
  LNode_type *lkeys = key_table ? get_list_elements(*key_table) : NULL;
  long tsearch = LOGR_get_total_number_searches(msn);
  int tnkeys = lkeys ? count_llist(lkeys) : 0;

  LNode_type *lusers = LOGR_get_list_users(msn);
  int tusers = lusers ? count_llist(lusers) : 0;
  short cus = LOGR_get_compute_user_sessions(msn);
  int ninobj = count_llist(get_list_objects(gf));
  int ext_log = LOGR_get_logs_filetype(msn);

  int tnsites = lsites ? count_llist(lsites) : 0;
  double tsize = LOGR_get_total_site_size(msn);
  long tacc = LOGR_get_total_site_naccesses(msn);

  int nhtml = 0;
  int nonhtml = 0;
  int nobj = 0;
  int nhyp_html = 0;
  int nhyp_nhtml = 0;
  int nehtml = 0;
  int nenhtml = 0;
  int nhost = 0;
  int nsa = 0;
  double arh=0, ard=0,ash=0,asd=0;

  LOGR_get_total_number_pages(msn,gf,&nhtml,&nonhtml,&nobj);
  LOGR_get_total_number_hyperlinks(msn,gf,&nhyp_html,&nhyp_nhtml);
  LOGR_get_total_number_entry_pages(msn,gf,&nehtml,&nenhtml);
  LOGR_get_total_number_referers(msn,gf,&nhost,&nsa);
  LOGR_get_average_by_time(msn,&ard,&arh,&asd,&ash);

  if(!(cus && ext_log == LOGR_EXTENDED_LOG))
      tusers = -1;

  if(ninobj <= 0)
      nobj = -1;

  GR_write_start_summary_logml_file(fp,tacc,tusers,tsize,nhtml,nonhtml,nobj,
			      nhyp_html,nhyp_nhtml,nehtml,nenhtml,tnsites,
			      nhost,nsa,tnurlextref,tnurlintref,tnuas,arh,
			      ard, ash, asd, tsearch,tnkeys);

  write_http_codes_logml_file(fp,msn);
  write_http_methods_logml_file(fp,msn);
  write_http_versions_logml_file(fp,msn);
  write_date_stats_logml_file(fp,msn);

  GR_write_end_summary_logml_file(fp);
  return 0;
}

int
write_path_edge_logml_file(FILE *fp, GraphFrame *gf,WebStatPtr msn,
			   WebGenLinkPtr gln, time_t t)
{
    int sid = LOGR_get_gen_link_source_num(gln);
    int tid = LOGR_get_gen_link_target_num(gln);
    char stime[100];
    HTBufferTimeApache(t,stime,100);
    GR_write_empty_uedge_logml_file(fp, sid, tid, stime);
    return 0;
}

int
write_user_path_logml_file(FILE *fp, GraphFrame *gf,WebStatPtr msn,
			      WebUserPtr usn)
{
    LNode_type *ltimes = LOGR_get_user_ltimes(usn);
    WebPathPtr pn = LOGR_get_user_current_path(usn);
    LNode_type *ledges = LOGR_get_path_ledges(pn);
    LNode_type *ptr = NULL, *qtr = NULL;
    int count = LOGR_get_path_number_edges(pn);

    if(!ledges || !ltimes)
	return -1;

    GR_write_start_path_logml_file(fp,count);
    
    qtr = ltimes->back;
    for(ptr = ledges->back; ptr != ledges; ptr=ptr->back)
    {
	time_t t = (time_t)qtr->info;
	WebGenLinkPtr gln = (WebGenLinkPtr)ptr->info;
	write_path_edge_logml_file(fp,gf,msn,gln,t);
	qtr = qtr->back;
    }
    GR_write_end_path_logml_file(fp);
    return 0;
}

int
write_user_session_logml_file(FILE *fp, GraphFrame *gf,WebStatPtr msn,
			      WebUserPtr usn)
{
    char *name = LOGR_get_user_name(usn);
    char *referer = LOGR_get_user_referer(usn);
    char *xmlescref = NULL;
    long acount = LOGR_get_user_naccesses(usn);
    char *epage = LOGR_get_user_entry_page(usn);
    char *xmlescepage = NULL;
    time_t t = LOGR_get_user_start_time(usn);
    char stime[100];
    HTBufferTimeApache(t,stime,100);
    xmlescref = referer ? LOGR_get_xml_nescape_string(referer): NULL;
    xmlescepage = epage ? LOGR_get_xml_nescape_string(epage): NULL;
    
    GR_write_start_user_session_logml_file(fp,name,xmlescref,xmlescepage,
					   stime,acount);
    write_user_path_logml_file(fp,gf,msn,usn);
    GR_write_end_user_session_logml_file(fp);

    if(xmlescref) {	
	free(xmlescref); xmlescref = NULL;
    }

    if(xmlescepage) {	
	free(xmlescepage); xmlescepage = NULL;
    }
    return 0;
}

int
write_user_sessions_logml_file(FILE *fp, GraphFrame *gf,WebStatPtr msn,
			       int count,int max_edges,int min_edges)
{
    int tnedges = 0;
    int tn = 0;
    int i=0;

    /*LNode_type *lusers = LOGR_get_list_users(msn);*/
    LNode_type *ptr = NULL;
    WebPathPtr *varray = NULL;
    LNode_type *lpaths = LOGR_get_list_paths(msn);


    for(ptr = lpaths->back; ptr != lpaths; ptr=ptr->back)
    {
	WebPathPtr gn = (WebPathPtr)ptr->info;

	int nedges = LOGR_get_path_number_edges(gn);

	if(nedges <= max_edges && nedges >= min_edges)
	    tnedges++;
    } 

    
    varray = tnedges ? (WebPathPtr*)calloc(tnedges,
					   sizeof(WebPathPtr)) : NULL;

    if(!varray && tnedges > 0) {
	fprintf(stderr,"Out of Memory write_user_sessions_logml_file\n");
	abort();
    }

    for(ptr = lpaths->back; ptr != lpaths; ptr=ptr->back)
    {
	WebPathPtr gn = (WebPathPtr)ptr->info;
	int nedges = LOGR_get_path_number_edges(gn);

	if(nedges <= max_edges && nedges >= min_edges)
	    varray[i++]=gn;
    } 

    if(tnedges <= 0) 
	return -1;

    i = 0;

    if(count > tnedges) 
	tn = tnedges;
    else
	tn = count;

    GR_write_start_user_sessions_logml_file(fp, tn, max_edges,min_edges);

    for(i = 1; i<=tn ; i++)
    {

	WebPathPtr gn = (WebPathPtr)randomized_select((GPTR *)varray,
							  i-1,tnedges-1,1,
					   LOGR_is_path_number_edges_lower,
				       	  LOGR_is_path_number_edges_bigger);

	WebUserPtr usn = gn ? (WebUserPtr)LOGR_get_path_info(gn) : NULL;

	write_user_session_logml_file(fp,gf,msn,usn);
    }

    GR_write_end_user_sessions_logml_file(fp);

    free(varray); varray = NULL;
    return 0;
}


